import threading
import time
import pyarrow as pa
import pyarrow.parquet as pq
import pandas as pd

# Create a sample table once
df = pd.DataFrame({
    'column1': range(1000000),
    'column2': range(1000000)
})
table = pa.Table.from_pandas(df)

# Function to write a Parquet file (just writing, no data creation)
def write_parquet_file(filename):
    pq.write_table(table, filename)

# Function to run multiple threads writing parquet files
def run_test_multithreaded():
    threads = []
    start = time.time()
    
    # Create 4 threads to write parquet files concurrently
    for i in range(12):
        thread = threading.Thread(target=write_parquet_file, args=(f'output_{i}.parquet',))
        threads.append(thread)
        thread.start()
    
    for thread in threads:
        thread.join()
    
    end = time.time()
    print(f"Multithreaded Time: {end - start:.4f} seconds")

# Function to run single-threaded test for comparison
def run_test_singlethreaded():
    start = time.time()
    
    # Write parquet files sequentially
    for i in range(12):
        write_parquet_file(f'output_single_{i}.parquet')
    
    end = time.time()
    print(f"Single-threaded Time: {end - start:.4f} seconds")

# Run both tests
run_test_singlethreaded()
run_test_multithreaded()

Single-threaded Time: 0.3169 seconds
Multithreaded Time: 0.0751 seconds